#coding:utf-8
from RedisClient import RedisClient
from proxySpider import Crawler

POOL_MAX_THRESHOLD = 10000

class Getter:
    def __init__(self):
        self.redis = RedisClient()
        self.crawler = Crawler()

    def is_over_threshold(self):
        '''
            作用：判断代理池是否到达极限
            :return 返回判断结果
        '''
        if self.redis.count() >= POOL_MAX_THRESHOLD:
            return True
        else:
            return False

    def run(self):
        '''
            启动代理获取器
        '''
        print('代理获取器开始执行..')
        if not self.is_over_threshold():
            for method in self.crawler.methods:
                if method[0:5] == 'crawl':
                    proxies = self.crawler.get_proxies(method)
                    for proxy in proxies:
                        self.redis.add(proxy)
           

if __name__ == '__main__':
    Getter().run()
